function [Xraw, Yraw, instanceID, ratingRow, data, histos ] = GetRadData(minAgreement)
%getRadData Generates the various X, Y matrices based on the radiologists
%   Can be tweaked based on the minimum number of radiologists giving a
%   rating for the data to be kept. Gets the feature data and makes an X
%   matrix, gets the rating data and makes a Y matrix. Also gives you a
%   histogram distribution of the ratings. 

%Load data
fprintf('Reading excel file\n'); %This read is the slow part
[~, ~, rawData] = xlsread('C:\Users\esmith2\Documents\Ethans Code\gitCode\Semantic Features\LIDC_All_Radiologist_Cases_For_Largest_Slice cleaned modified ES recalc Features by PS.xlsx');

ratingColumn = [9,11,12,13,14,15,16]; %columns where the ratings we want are found
numCategories = length(ratingColumn);

%Get colums of special headers
for i = 1:size(rawData,2)
    if strcmp(rawData{1,i}, 't1.coords') == 1
        featureColumn = i + 1;
    end
end
numFeatures = size(rawData,2) - (featureColumn - 1);

%[~, ~, rawData] = xlsread('C:\Ethan\Dropbox\MedIX\Lung Segmentation\Repo\Semantic Features\cleaned data in order with features.xlsx');
rawData = rawData(2:end, :); %Remove headers 

%Sort numRatings
fprintf('Only looking at data with at least %d ratings\n' , minAgreement);
currentID = rawData(1,6);
dupCount = 1;
goodRows = cell(1,4);
rowStart = 0;
rowStop = 0;
for i = 2:size(rawData,1)
    %fprintf('%d %d %d %d\n', i, strcmp(currentID, rawData(i,6)), dupCount, minAgreement);
    if strcmp(currentID, rawData(i,6)) == 1 %Still part of existing line
        dupCount = dupCount + 1;
    else %Found new set, process it
        rowStart = rowStop + 1;
        rowStop = i-1;
        goodRows{dupCount} = vertcat(goodRows{dupCount}, [rowStart:rowStop]');

        currentID = rawData(i,6);%start new count
        dupCount = 1;
    end
end
%process last set
rowStart = rowStop + 1;
rowStop = i;
goodRows{dupCount} = vertcat(goodRows{dupCount}, [rowStart:rowStop]');

%form sorted data   
data = rawData(goodRows{4},:);
for i = 3:-1:minAgreement
    data = vertcat(data, rawData(goodRows{i},:) );
end
%fprintf('4 %d 3 %d 2 %d 1 %d Total %d\n', length(goodRows{4}), length(goodRows{3}), length(goodRows{2}), length(goodRows{1}), length(goodRows{4}) + length(goodRows{3}) + length(goodRows{2}) + length(goodRows{1}));

%Get indexes for where each group starts and stops (all the 4s, all the
%3s etc..
ratingRow = zeros(1,4);
ratingRowGroup = zeros(1,4);
ratingRow(1,4) = 1;
ratingRowGroup(1,4) = 1;
for i = 3:-1:1
    ratingRow(1,i)      = length(goodRows{i+1}) + ratingRow(1,i+1);
    ratingRowGroup(1,i) = length(goodRows{i+1}) / (i+1) + ratingRowGroup(1,i+1);
end
%Get instance IDs for each row so we can match up with features
%extracted in a different order
instanceID = cell2mat(data(:,1));

fprintf('Selecting all data\n'); 
%Get strait one to one values
Xraw = cell2mat(data(:,featureColumn:featureColumn+numFeatures-1));
Yraw = cell2mat(data(:,ratingColumn));

%Make histograms for check against broken clock/noise
%tabulate(Ytest(:,1)) function is better :-(
histos = zeros(numCategories, 5);
for i = 1:numCategories
    YVectors = Y2YVectors(Yraw(:,i), 5);
    histos(i,:) = sum(YVectors)/ sum(sum(YVectors));
end
%histos = [histos, zeros(numCategories, 1)];
%histos = horzcat(histos, max(histos,[], 2));

clear localDirectoryName serverDirectoryName lidcDirectoryName directoryName dataFileFullName YVectors ...
    categories currentID dupCount featureColumn featureDirectory i message numCategories 
    
%clear rawData goodRows
end